package us.codecraft.webmagic.processor.example;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.ResultItems;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;

/***
 * <p>
 *   此参数用于爬取百度百科
 * </p>
 *
 * @version 1.0
 * @author feilong
 */
public class BaiduBaikePageProcessorExtension implements PageProcessor {

    private Site site = Site.me()
            //设置重试次数为3次
            .setRetryTimes(3)
            //设置抓取间隔为一秒
            .setSleepTime(1000)
            .setUseGzip(true);

    public static void main(String[] args) {
        Spider spider = Spider.create(new BaiduBaikePageProcessorExtension()).thread(2);
        String urlTemplate = "https://baike.baidu.com/wikicategory/view?categoryName=%s";
        List<String> list = new ArrayList<String>(5);
        list.add(String.format(urlTemplate, "恐龙大全"));
        list.add(String.format(urlTemplate, "多肉植物"));
        List<ResultItems> resultItems = spider.<ResultItems>getAll(list);
        for (ResultItems resultItem : resultItems) {
            System.out.println(resultItem.getAll());
        }
        spider.close();
    }

    @Override
    public void process(Page page) {
        page.putField("title", page.getHtml()
                .css("div.head-body__side__content1 ul li div.content__title",
                        "text"));
        page.putField("description", page.getHtml()
                .css("div.head-body__side__content1 ul li div.content__tag",
                        "text"));
        String linkUrl = page.getHtml().css("div.head-body__side__content1 ul li a",
                "href").get();
        page.putField("linkUrl", encodeUrl(linkUrl));
    }

    private String encodeUrl(String url) {
        if (url != null) {
            try {
                return URLDecoder.decode(url, StandardCharsets.UTF_8);
            } catch (IllegalArgumentException | NullPointerException npe) {
                System.err.println("decode url error,please check!");
            }
        }
        return "";
    }

    @Override
    public Site getSite() {
        return site;
    }
}
